{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pprint as pp\n",
    "import logging\n",
    "import keras.backend as K\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers import GRU, Input, TimeDistributed, BatchNormalization\n",
    "from keras.layers import Bidirectional, Dense, Activation,Conv1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "the_input (InputLayer)       (None, None, 161)         0         \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, None, 1024)        1814528   \n",
      "_________________________________________________________________\n",
      "bn_conv1d (BatchNormalizatio (None, None, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "rnn_1 (GRU)                  (None, None, 1024)        6294528   \n",
      "_________________________________________________________________\n",
      "rnn_2 (GRU)                  (None, None, 1024)        6294528   \n",
      "_________________________________________________________________\n",
      "rnn_3 (GRU)                  (None, None, 1024)        6294528   \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, None, 29)          29725     \n",
      "=================================================================\n",
      "Total params: 20,731,933\n",
      "Trainable params: 20,729,885\n",
      "Non-trainable params: 2,048\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.engine.training.Model at 0x1364dd710>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def gru_model(input_dim=161, output_dim=29, recur_layers=3, nodes=1024,\n",
    "              kernel_size=11, conv_border_mode='valid',  conv_stride=2,\n",
    "              initialization='glorot_uniform', batch_norm=True):\n",
    "    \"\"\"\n",
    "    Building a recurrent neatual network (CTC) for speech\n",
    "    with GRU units.\n",
    "    \"\"\"\n",
    "    acoustic_input = Input(shape=(None, input_dim), name='the_input')\n",
    "    \n",
    "    conv_1d = Conv1D(nodes, kernel_size, padding=conv_border_mode,\n",
    "                           strides=conv_stride, kernel_initializer=initialization,\n",
    "                           activation='relu', name='conv1d')(acoustic_input)\n",
    "    \n",
    "    output = BatchNormalization(name='bn_conv1d')(conv_1d) if batch_norm else conv1d\n",
    "    \n",
    "    for i in range(recur_layers):\n",
    "        output = GRU(nodes, activation='relu', kernel_initializer=initialization,\n",
    "                    return_sequences=True, name='rnn_{}'.format(i+1))(output)\n",
    "        \n",
    "        if batch_norm:\n",
    "            bn_layer = BatchNormalization(name='bn_rnn_{}'.format(i+1))\n",
    "            otuput = bn_layer(output)\n",
    "            \n",
    "    y_pred = TimeDistributed(Dense(output_dim, activation='linear', \n",
    "                                  kernel_initializer=initialization, name='dense'))(output)\n",
    "    \n",
    "    model = Model(inputs=acoustic_input, outputs=y_pred)\n",
    "    model.output_length = lambda x:x\n",
    "    print(model.summary())\n",
    "    return model\n",
    "    \n",
    "gru_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
